{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "from tqdm import tqdm\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import xgboost\n",
    "import lightgbm\n",
    "import catboost\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import classification_report, f1_score,precision_score,recall_score,accuracy_score\n",
    "from sklearn.model_selection import StratifiedKFold, KFold\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from matplotlib import pyplot as plt\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "import pandas as pd\n",
    "import gc\n",
    "from sklearn import feature_extraction\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.ensemble import (RandomForestClassifier, AdaBoostClassifier, \n",
    "                              GradientBoostingClassifier, ExtraTreesClassifier)\n",
    "from category_encoders.woe import WOEEncoder\n",
    "from category_encoders.target_encoder import TargetEncoder\n",
    "from category_encoders.cat_boost import CatBoostEncoder\n",
    "from category_encoders.ordinal import OrdinalEncoder\n",
    "from category_encoders.leave_one_out import LeaveOneOutEncoder\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "plt.rcParams['font.sans-serif']=['Simhei']\n",
    "plt.rcParams['axes.unicode_minus']=False\n",
    "import json\n",
    "import jieba\n",
    "from scipy.stats import entropy\n",
    "import fasttext"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 0 数据的简单分析"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "base_info shape: (24865, 33) id unique: 24865\n",
      "annual_report_info shape: (22550, 23) id unique: 8937\n",
      "tax_info shape: (29195, 9) id unique: 808\n",
      "change_info shape: (45940, 5) id unique: 8726\n",
      "news_info shape: (10518, 3) id unique: 927\n",
      "other_info shape: (1890, 4) id unique: 1888\n",
      "entprise_info shape: (14865, 2) id unique: 14865\n",
      "entprise_evaluate shape: (10000, 2) id unique: 10000\n"
     ]
    }
   ],
   "source": [
    "base_info=pd.read_csv('../train/base_info.csv')#企业的基本信息\n",
    "annual_report_info=pd.read_csv('../train/annual_report_info.csv')#企业的年报基本信息\n",
    "tax_info=pd.read_csv('../train/tax_info.csv')#企业的纳税信息\n",
    "change_info=pd.read_csv('../train/change_info.csv')#变更信息\n",
    "news_info=pd.read_csv('../train/news_info.csv')#舆情信息\n",
    "other_info=pd.read_csv('../train/other_info.csv')#其它信息\n",
    "entprise_info=pd.read_csv('../train/entprise_info.csv')#企业标注信息{0: 13884, 1: 981}\n",
    "entprise_evaluate=pd.read_csv('../entprise_evaluate.csv')#未标注信息\n",
    "\n",
    "print('base_info shape:',base_info.shape,'id unique:',len(base_info['id'].unique()))\n",
    "print('annual_report_info shape:',annual_report_info.shape,'id unique:',len(annual_report_info['id'].unique()))\n",
    "print('tax_info shape:',tax_info.shape,'id unique:',len(tax_info['id'].unique()))\n",
    "print('change_info shape:',change_info.shape,'id unique:',len(change_info['id'].unique()))\n",
    "print('news_info shape:',news_info.shape,'id unique:',len(news_info['id'].unique()))\n",
    "print('other_info shape:',other_info.shape,'id unique:',len(other_info['id'].unique()))\n",
    "print('entprise_info shape:',entprise_info.shape,'id unique:',len(entprise_info['id'].unique()))\n",
    "print('entprise_evaluate shape:',entprise_evaluate.shape,'id unique:',len(entprise_evaluate['id'].unique()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1 特征构建 \n",
    "###  tfidi处理经营范围(opscope)特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "对opscope提取tfidif特征完毕..........\n"
     ]
    }
   ],
   "source": [
    "# tfidif 处理经营范围的特征\n",
    "#cn_stopwords.txt来源于 https://github.com/goto456/stopwords\n",
    "def stopwordslist():\n",
    "    stopwords = [line.strip() for line in open('D:/tianma/stopwords-master/cn_stopwords.txt',encoding='UTF-8').readlines()]\n",
    "    return stopwords\n",
    "# 创建一个停用词列表\n",
    "stopwords = stopwordslist()\n",
    "stopwords+=['、', '；', '，', '）','（']\n",
    "#\n",
    "train_df_scope=base_info.merge(entprise_info)[['id','opscope','label']]\n",
    "test_df_scope=base_info[base_info['id'].isin(entprise_evaluate['id'].unique().tolist())]\n",
    "test_df_scope=test_df_scope.reset_index(drop=True)[['id','opscope']]\n",
    "str_label_0=''\n",
    "str_label_1=''\n",
    "for index,name,opscope,label in train_df_scope.itertuples():\n",
    "    # 结巴分词\n",
    "    seg_text = jieba.cut(opscope.replace(\"\\t\", \" \").replace(\"\\n\", \" \"))\n",
    "    outline = \" \".join(seg_text)\n",
    "    out_str=\"\"\n",
    "    for per in outline.split():\n",
    "        if per not in stopwords: \n",
    "            out_str += per\n",
    "            out_str+=\" \"\n",
    "    if label==0:\n",
    "        str_label_0+=out_str\n",
    "    else:\n",
    "        str_label_1+=out_str\n",
    "corpus=[str_label_0,str_label_1]\n",
    "vectorizer=CountVectorizer()#该类会将文本中的词语转换为词频矩阵，矩阵元素a[i][j] 表示j词在i类文本下的词频\n",
    "transformer=TfidfTransformer()#该类会统计每个词语的tf-idf权值\n",
    "tfidf=transformer.fit_transform(vectorizer.fit_transform(corpus))#第一个fit_transform是计算tf-idf，第二个fit_transform是将文本转为词频矩阵\n",
    "word=vectorizer.get_feature_names()#获取词袋模型中的所有词语总共7175个词语\n",
    "weight=tfidf.toarray()#将(2, 7175)tf-idf矩阵抽取出来，元素a[i][j]表示j词在i类文本中的tf-idf权重\n",
    "# for i in range(len(weight)):#打印每类文本的tf-idf词语权重，第一个for遍历所有文本，第二个for便利某一类文本下的词语权重\n",
    "#     #\n",
    "#     for j in range(len(word)):\n",
    "#         print(word[j],weight[i][j])\n",
    "#下面将会根据tfidi算出来的权重将经营范围的文本特征转换为数值(利用weight[1,:]也即各个词语在第二类(违法类中所占据的权重之和))\n",
    "illegal_word_weights={}\n",
    "for i in range(len(word)):\n",
    "    illegal_word_weights[word[i]]=weight[1][i]\n",
    "tfidi_opscope=[]\n",
    "for index,name,opscope in base_info[['id','opscope']].itertuples():\n",
    "    # \n",
    "    seg_text = jieba.cut(opscope.replace(\"\\t\", \" \").replace(\"\\n\", \" \"))\n",
    "    outline = \" \".join(seg_text)\n",
    "    tfidi_frt=0\n",
    "    for per in outline.split():\n",
    "        if per in illegal_word_weights: \n",
    "            tfidi_frt+=illegal_word_weights[per]\n",
    "    tfidi_opscope.append(tfidi_frt)\n",
    "base_info['tfidif_opscope']=tfidi_opscope\n",
    "print('对opscope提取tfidif特征完毕..........')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  change_info、other_info，news_info，annual_report_info,tax表格的简单特征构建"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "finished .............\n"
     ]
    }
   ],
   "source": [
    "#change_info\n",
    "change_info_clean=change_info.drop(['bgrq','bgq','bgh'],axis=1)\n",
    "change_info_clean = change_info_clean.groupby('id',sort=False).agg('mean')\n",
    "change_info_clean=pd.DataFrame(change_info_clean).reset_index()\n",
    "#other_info\n",
    "#空值大于0.5的列都删除掉\n",
    "buf_group = other_info.groupby('id',sort=False).agg('mean')\n",
    "other_info_clean=pd.DataFrame(buf_group).reset_index()\n",
    "other_info_clean=other_info_clean.fillna(-1)\n",
    "other_info_clean = other_info_clean.groupby('id',sort=False).agg('mean')\n",
    "other_info_clean=pd.DataFrame(other_info_clean).reset_index()\n",
    "#news_info\n",
    "news_info_clean=news_info.drop(['public_date'],axis=1)\n",
    "#对object类型进行编码\n",
    "news_info_clean['positive_negtive']=news_info_clean['positive_negtive'].fillna(\"中立\")\n",
    "#\n",
    "dic={}\n",
    "cate=news_info_clean.positive_negtive.unique()\n",
    "for i in range(len(cate)):\n",
    "    dic[cate[i]]=i\n",
    "#\n",
    "news_info_clean['positive_negtive']=news_info_clean['positive_negtive'].map(dic)\n",
    "news_info_clean = news_info_clean.groupby('id',sort=False).agg('mean')\n",
    "news_info_clean=pd.DataFrame(news_info_clean).reset_index()\n",
    "#处理annual_report_info的数据\n",
    "#空值大于0.5的列都删除掉\n",
    "annual_report_info_clean=annual_report_info.dropna(thresh=annual_report_info.shape[0]*0.5,how='all',axis=1)\n",
    "#对object类型进行编码\n",
    "annual_report_info_clean['BUSSTNAME']=annual_report_info_clean['BUSSTNAME'].fillna(\"无\")\n",
    "dic = {'无':-1,'开业':0, '歇业':1, '停业':2, '清算':3}\n",
    "#\n",
    "annual_report_info_clean['BUSSTNAME']=annual_report_info_clean['BUSSTNAME'].map(dic)\n",
    "annual_report_info_clean = annual_report_info_clean.groupby('id',sort=False).agg('mean')\n",
    "annual_report_info_clean=pd.DataFrame(annual_report_info_clean).reset_index()\n",
    "#处理tax数据\n",
    "tax_info_clean=tax_info.copy()\n",
    "tax_info_clean['START_DATE']=pd.to_datetime(tax_info_clean['START_DATE'])\n",
    "tax_info_clean['END_DATE']=pd.to_datetime(tax_info_clean['END_DATE'])\n",
    "tax_info_clean['gap_day']=(tax_info_clean['END_DATE']-tax_info_clean['START_DATE']).dt.total_seconds()//3600//24\n",
    "tax_info_clean=tax_info_clean.drop(['START_DATE','END_DATE'],axis=1)\n",
    "tax_info_clean['TAX_CATEGORIES']=tax_info_clean['TAX_CATEGORIES'].fillna(\"无\")#17 unique\n",
    "tax_info_clean['TAX_ITEMS']=tax_info_clean['TAX_ITEMS'].fillna(\"无\")#275 TAX_ITEMS\n",
    "#对object类型进行编码\n",
    "dic={}\n",
    "cate=tax_info_clean.TAX_CATEGORIES.unique()\n",
    "for i in range(len(cate)):\n",
    "    dic[cate[i]]=i\n",
    "tax_info_clean['TAX_CATEGORIES']=tax_info_clean['TAX_CATEGORIES'].map(dic)\n",
    "#\n",
    "dic={}\n",
    "cate=tax_info_clean.TAX_ITEMS.unique()\n",
    "for i in range(len(cate)):\n",
    "    dic[cate[i]]=i\n",
    "tax_info_clean['TAX_ITEMS']=tax_info_clean['TAX_ITEMS'].map(dic)\n",
    "tax_info_clean['income']=tax_info_clean['TAX_AMOUNT']/tax_info_clean['TAX_RATE']\n",
    "#\n",
    "tax_info_clean = tax_info_clean.groupby('id',sort=False).agg('mean')\n",
    "tax_info_clean=pd.DataFrame(tax_info_clean).reset_index()\n",
    "#税额分箱\n",
    "tax_info_clean['TAX_AMOUNT']=tax_info_clean['TAX_AMOUNT'].fillna(tax_info_clean['TAX_AMOUNT'].median())\n",
    "tax_info_clean['bucket_TAX_AMOUNT']=pd.qcut(tax_info_clean['TAX_AMOUNT'], 10, labels=False,duplicates='drop')\n",
    "print('finished .............')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## base_info数据较为重要，需要构建诸多交叉特征以及特征分箱"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|                                                                                        | 0/24865 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "编码完毕.................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████| 24865/24865 [00:00<00:00, 164022.43it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████| 24865/24865 [00:00<00:00, 158843.45it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████| 24865/24865 [00:00<00:00, 159817.11it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████| 24865/24865 [00:00<00:00, 161892.84it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████| 24865/24865 [00:00<00:00, 165650.71it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████| 24865/24865 [00:00<00:00, 164023.21it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(24865, 42)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# #处理base_info数据\n",
    "base_info['opto']=pd.to_datetime(base_info['opto']).fillna(pd.to_datetime(base_info['opto']).max())\n",
    "base_info_clean=base_info.drop(['opscope','opto'],axis=1)\n",
    "base_info_clean['opfrom']=pd.to_datetime(base_info_clean['opfrom'])\n",
    "current_time=pd.to_datetime('2020-11-17 00:00:00')\n",
    "base_info_clean['alive_year']=(current_time-pd.to_datetime(base_info_clean['opfrom'])).dt.total_seconds()//3600//24//30\n",
    "base_info_clean.drop(['opfrom'],axis=1,inplace=True)\n",
    "#............................对object类型进行编码...............................\n",
    "base_info_clean['industryphy']=base_info_clean['industryphy'].fillna(\"无\")\n",
    "base_info_clean['dom']=base_info_clean['dom'].fillna(\"无\")\n",
    "base_info_clean['opform']=base_info_clean['opform'].fillna(\"无\")\n",
    "base_info_clean['oploc']=base_info_clean['oploc'].fillna(\"无\")\n",
    "#\n",
    "dic={}\n",
    "cate=base_info_clean.industryphy.unique()\n",
    "for i in range(len(cate)):\n",
    "    dic[cate[i]]=i\n",
    "base_info_clean['industryphy']=base_info_clean['industryphy'].map(dic)\n",
    "#\n",
    "dic={}\n",
    "cate=base_info_clean.dom.unique()\n",
    "for i in range(len(cate)):\n",
    "    dic[cate[i]]=i\n",
    "base_info_clean['dom']=base_info_clean['dom'].map(dic)\n",
    "#\n",
    "dic={}\n",
    "cate=base_info_clean.opform.unique()\n",
    "for i in range(len(cate)):\n",
    "    dic[cate[i]]=i\n",
    "base_info_clean['opform']=base_info_clean['opform'].map(dic)\n",
    "#\n",
    "dic={}\n",
    "cate=base_info_clean.oploc.unique()\n",
    "for i in range(len(cate)):\n",
    "    dic[cate[i]]=i\n",
    "base_info_clean['oploc']=base_info_clean['oploc'].map(dic)\n",
    "#\n",
    "base_info_clean=base_info_clean.fillna(-1)\n",
    "#\n",
    "print('编码完毕.................')\n",
    "#........................分箱.................................\n",
    "def bucket(name,bucket_len):\n",
    "    gap_list=[base_info_clean[name].quantile(i/bucket_len) for i in range(bucket_len+1)]#以分位数作为分箱标志\n",
    "    len_data=len(base_info_clean[name])\n",
    "    new_col=[]\n",
    "    for i in base_info_clean[name].values:\n",
    "        for j in range(len(gap_list)):\n",
    "            if gap_list[j]>=i:\n",
    "                encode=j\n",
    "                break\n",
    "        new_col.append(encode)\n",
    "    return new_col\n",
    "#注册资本_实缴资本\n",
    "base_info_clean['regcap_reccap']=base_info_clean['regcap']-base_info_clean['reccap']\n",
    "#注册资本分箱\n",
    "base_info_clean['regcap']=base_info_clean['regcap'].fillna(base_info_clean['regcap'].median())\n",
    "base_info_clean['bucket_regcap']=pd.qcut(base_info_clean['regcap'], 10, labels=False,duplicates='drop')\n",
    "#实缴资本分箱\n",
    "base_info_clean['reccap']=base_info_clean['reccap'].fillna(base_info_clean['reccap'].median())\n",
    "base_info_clean['bucket_reccap']=pd.qcut(base_info_clean['reccap'], 10, labels=False,duplicates='drop')\n",
    "#注册资本_实缴资本分箱\n",
    "base_info_clean['regcap_reccap']=base_info_clean['regcap_reccap'].fillna(base_info_clean['regcap_reccap'].median())\n",
    "base_info_clean['bucket_regcap_reccap']=pd.qcut(base_info_clean['regcap_reccap'], 10, labels=False,duplicates='drop')\n",
    "#.............................交叉.........................\n",
    "#作两个特征的交叉\n",
    "def cross_two(name_1,name_2):\n",
    "    new_col=[]\n",
    "    encode=0\n",
    "    dic={}\n",
    "    val_1=base_info_clean[name_1]\n",
    "    val_2=base_info_clean[name_2]\n",
    "    for i in tqdm(range(len(val_1))):\n",
    "        tmp=str(val_1[i])+'_'+str(val_2[i])\n",
    "        if tmp in dic:\n",
    "            new_col.append(dic[tmp])\n",
    "        else:\n",
    "            dic[tmp]=encode\n",
    "            new_col.append(encode)\n",
    "            encode+=1\n",
    "    return new_col\n",
    "#作企业类型-小类的交叉特征\n",
    "base_info_clean['enttypegb']=base_info_clean['enttypegb'].fillna(\"无\")\n",
    "base_info_clean['enttypeitem']=base_info_clean['enttypeitem'].fillna(\"无\")\n",
    "new_col=cross_two('enttypegb','enttypeitem')#作企业类型-小类的交叉特征\n",
    "base_info_clean['enttypegb_enttypeitem']=new_col\n",
    "#\n",
    "#行业类别-细类的交叉特征\n",
    "base_info_clean['industryphy']=base_info_clean['industryphy'].fillna(\"无\")\n",
    "base_info_clean['industryco']=base_info_clean['industryco'].fillna(\"无\")\n",
    "new_col=cross_two('industryphy','industryco')#作企业类型-小类的交叉特征\n",
    "base_info_clean['industryphy_industryco']=new_col\n",
    "#企业类型-行业类别的交叉特征\n",
    "new_col=cross_two('enttypegb','industryphy')#作企业类型-小类的交叉特征\n",
    "base_info_clean['enttypegb_industryphy']=new_col\n",
    "#行业类别-企业类型小类的交叉特征\n",
    "new_col=cross_two('industryphy','enttypeitem')#作企业类型-小类的交叉特征\n",
    "base_info_clean['industryphy_enttypeitem']=new_col\n",
    "#行业类别细类--企业类型小类的交叉特征\n",
    "new_col=cross_two('industryco','enttypeitem')#作企业类型-小类的交叉特征\n",
    "base_info_clean['industryco_enttypeitem']=new_col\n",
    "\n",
    "#企业类型-小类-行业类别-细类的交叉特征\n",
    "new_col=cross_two('enttypegb_enttypeitem','industryphy_industryco')#作企业类型-小类的交叉特征\n",
    "base_info_clean['enttypegb_enttypeitem_industryphy_industryco']=new_col\n",
    "base_info_clean.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## category特征单独提取出来"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "cat_features=['industryphy','dom','opform','oploc','bucket_regcap',\n",
    "              'bucket_reccap','bucket_regcap_reccap',\n",
    "              'enttypegb','enttypeitem','enttypegb_enttypeitem',\n",
    "              'enttypegb_industryphy','enttypegb_enttypeitem_industryphy_industryco',\n",
    "              'industryphy','industryco','industryphy_industryco',\n",
    "              'industryphy_enttypeitem','industryco_enttypeitem',\n",
    "              'adbusign','townsign','regtype','TAX_CATEGORIES','bucket_TAX_AMOUNT',\n",
    "              'legal_judgment_num','brand_num','patent_num','positive_negtive'\n",
    "             ]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(24865, 73)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#\n",
    "all_data=base_info_clean.merge(annual_report_info_clean,how='outer')\n",
    "all_data=all_data.merge(tax_info_clean,how='outer')\n",
    "all_data=all_data.merge(change_info_clean,how='outer')\n",
    "all_data=all_data.merge(news_info_clean,how='outer')\n",
    "all_data=all_data.merge(other_info_clean,how='outer')\n",
    "all_data=all_data.fillna(-1)\n",
    "all_data[cat_features]=all_data[cat_features].astype(int)\n",
    "all_data.shape#,base_info.shape,annual_report_info.shape,tax_info.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((14865, 72), (10000, 72))"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#\n",
    "train_df=all_data.merge(entprise_info)\n",
    "train_data=train_df.drop(['id','label'],axis=1)\n",
    "kind=train_df['label']\n",
    "test_df=all_data[all_data['id'].isin(entprise_evaluate['id'].unique().tolist())]\n",
    "test_df=test_df.reset_index(drop=True)\n",
    "test_data=test_df.drop(['id'],axis=1)\n",
    "train_data.shape,test_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # 伪标签\n",
    "# use_pseudo=True\n",
    "# if use_pseudo:\n",
    "#     train_data=train_df.drop(['id','label'],axis=1)\n",
    "#     kind=train_df['label']\n",
    "#     pseudo_name=[]\n",
    "#     pseudo_label={'id':[],'label':[]}\n",
    "#     pseudo_df=pd.read_csv('submit_857_xgb_rf_lgb_cab.csv')\n",
    "#     for index,name,score in pseudo_df.itertuples():\n",
    "#         if score>0.9 or score<0.05:\n",
    "#             pseudo_label['id'].append(name)\n",
    "#             if score>0.9 :\n",
    "#                 pseudo_label['label'].append(1)\n",
    "#             else:\n",
    "#                 pseudo_label['label'].append(0)\n",
    "#             pseudo_name.append(name)\n",
    "#     len(pseudo_name)\n",
    "#     pseudo_data=test_df[test_df.id.isin(pseudo_name)].reset_index(drop=True)\n",
    "#     pseudo_data=pseudo_data.merge(pd.DataFrame(pseudo_label))\n",
    "#     #\n",
    "#     train_df=pd.concat((train_df,pseudo_data)).reset_index(drop=True)\n",
    "#     train_data=train_df.drop(['id','label'],axis=1)\n",
    "#     kind=train_df['label']\n",
    "#     print(train_data.shape,test_data.shape)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "#特征筛选\n",
    "# frt_select=[\n",
    "#  'industryphy',\n",
    "#  'enttypegb',\n",
    "#  'regcap',\n",
    "#  'townsign',\n",
    "#  'industryco',\n",
    "#  'bucket_regcap',\n",
    "#  'empnum',\n",
    "#  'bucket_reccap',\n",
    "#  'enttypeitem',\n",
    "#  'industryphy_industryco',\n",
    "#  'reccap',\n",
    "#  'FORINVESTSIGN',\n",
    "#  'positive_negtive',\n",
    "#  'regtype',\n",
    "#  'STOCKTRANSIGN',\n",
    "#  'bucket_regcap_reccap',\n",
    "#  'enttypegb_enttypeitem',\n",
    "#  'regcap_reccap',\n",
    "#  'legal_judgment_num',\n",
    "#  'TAX_CATEGORIES',\n",
    "#  'TAX_AMOUNT',\n",
    "#  'bgq_bgh',\n",
    "#  'TAX_ITEMS']\n",
    "# frt_select=important_frt[:30]\n",
    "# train_data=train_data[frt_select]\n",
    "# test_data=test_data[frt_select]\n",
    "# cat_features=list(set(frt_select).intersection(set(cat_features)))\n",
    "# cat_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def eval_score(y_test,y_pre):\n",
    "#     _,_,f_class,_=precision_recall_fscore_support(y_true=y_test,y_pred=y_pre,labels=[0,1],average=None)\n",
    "#     fper_class={'合法':f_class[0],'违法':f_class[1],'f1':f1_score(y_test,y_pre)}\n",
    "#     return fper_class\n",
    "#\n",
    "def eval_score(y_test,y_pre):\n",
    "    valid_f1 = f1_score(y_test, y_pre)\n",
    "    valid_p = precision_score(y_test, y_pre)\n",
    "    valid_r = recall_score(y_test, y_pre)\n",
    "    F = valid_p*0.7+valid_r*0.2+valid_f1*0.1\n",
    "    return F\n",
    "\n",
    "#\n",
    "def k_fold_serachParmaters(model,train_val_data,train_val_kind):\n",
    "    mean_f1=0\n",
    "    mean_f1Train=0\n",
    "    n_splits=5\n",
    "    sk = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=2020)\n",
    "    for train, test in sk.split(train_val_data, train_val_kind):\n",
    "        x_train = train_val_data.iloc[train]\n",
    "        y_train = train_val_kind.iloc[train]\n",
    "        x_test = train_val_data.iloc[test]\n",
    "        y_test = train_val_kind.iloc[test]\n",
    "\n",
    "        model.fit(x_train, y_train)\n",
    "        pred = model.predict(x_test)\n",
    "        weigted_score =  eval_score(y_test,pred)\n",
    "        mean_f1+=weigted_score/n_splits\n",
    "        #print(fper_class)\n",
    "        \n",
    "        pred_Train = model.predict(x_train)\n",
    "        weigted_score_train =  eval_score(y_train,pred_Train)\n",
    "        mean_f1Train+=weigted_score_train/n_splits\n",
    "    #print('mean valf1:',mean_f1)\n",
    "    #print('mean trainf1:',mean_f1Train)\n",
    "    return mean_f1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n_estimators: 150\n",
      "[150, 0.06, 7, 7] 0.8157825742455604\n",
      "[150, 0.06, 7, 9] 0.8206371265625654\n",
      "[150, 0.06, 8, 9] 0.823178846036303\n"
     ]
    }
   ],
   "source": [
    "def search_param(n_estimators,lr,max_depth,num_leaves):\n",
    "    params={'num_leaves':num_leaves\n",
    "                   ,'max_depth':max_depth\n",
    "                   ,'learning_rate':lr\n",
    "                   ,'n_estimators':n_estimators\n",
    "                   ,'n_jobs':8\n",
    "               }\n",
    "    mean_f1=k_fold_serachParmaters(lightgbm.LGBMClassifier(**params),train_data,kind)\n",
    "    return mean_f1\n",
    "\n",
    "#搜索最佳参数\n",
    "param=[]\n",
    "best=0\n",
    "for n_estimators in [150]:\n",
    "    print('n_estimators:',n_estimators)\n",
    "    for lr in [0.06,]:\n",
    "        for max_depth in [7,8,9,]:\n",
    "            for num_leaves in [7,9,11,13]:\n",
    "                mean_f1=search_param(n_estimators,lr,max_depth,num_leaves)\n",
    "                if mean_f1>best:\n",
    "                    param=[n_estimators,lr,max_depth,num_leaves]\n",
    "                    best=mean_f1\n",
    "                    print(param,best)#[150, 0.07, 13, 11] 0.8370422016216938"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "#\n",
    "def search_param(n_estimators,lr,max_depth,min_child_weight):\n",
    "    params={'max_depth':max_depth\n",
    "              ,'learning_rate':lr\n",
    "              ,'n_estimators':n_estimators\n",
    "              ,'min_child_weight':min_child_weight\n",
    "              ,'n_jobs':8\n",
    "              ,'importance_type':'total_cover'\n",
    "           }\n",
    "    \n",
    "    mean_f1=k_fold_serachParmaters(xgboost.XGBClassifier(**params),train_data,kind)\n",
    "    return mean_f1\n",
    "\n",
    "#搜索最佳参数\n",
    "param=[]\n",
    "# best=0\n",
    "# for n_estimators in [50]:\n",
    "#     print('n_estimators:',n_estimators)\n",
    "#     for lr in [0.05,0.07,0.09,0.1]:\n",
    "#         for max_depth in [7,8,9,10]:\n",
    "#             for min_child_weight in [7,9,11,13,15]:\n",
    "#                 mean_f1=search_param(n_estimators,lr,max_depth,min_child_weight)\n",
    "#                 if mean_f1>best:\n",
    "#                     param=[n_estimators,lr,max_depth,min_child_weight]\n",
    "#                     best=mean_f1\n",
    "#                     print(param,best)#[50, 0.04, 4, 13] 0.8449774193716448"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "iter_cnt: 100\n",
      "[100, 0.08, 8] 0.8209030188770057\n",
      "[100, 0.1, 8] 0.8312103438202738\n",
      "[100, 0.1, 8] 0.8312103438202738\n"
     ]
    }
   ],
   "source": [
    "def search_param(iter_cnt,lr,max_depth):\n",
    "    \n",
    "    params={'iterations':iter_cnt\n",
    "              ,'learning_rate':lr\n",
    "              ,'depth':max_depth\n",
    "              ,'silent':True\n",
    "              ,'thread_count':8\n",
    "              ,'task_type':'CPU'\n",
    "        }\n",
    "    mean_f1=k_fold_serachParmaters(catboost.CatBoostClassifier(**params),train_data,kind)\n",
    "    return mean_f1\n",
    "\n",
    "#搜索最佳参数[52, 0.073, 7] 0.8440488273317126\n",
    "param=[]\n",
    "best=0\n",
    "for iter_cnt in [100]:\n",
    "    print('iter_cnt:',iter_cnt)\n",
    "    for lr in [0.08,0.1]:\n",
    "        for max_depth in [8]:\n",
    "            mean_f1=search_param(iter_cnt,lr,max_depth)\n",
    "            if mean_f1>best:\n",
    "                param=[iter_cnt,lr,max_depth]#[55, 0.076, 7]\n",
    "                best=mean_f1\n",
    "                print(param,best)\n",
    "print(param,best)#筛选特征:[70, 0.06 8] 0.8417684642475657 所有特征:[54, 0.07, 7] 0.8411337269934891\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForestClassifier_v1: 0.8499826928084713\n",
      "RandomForestClassifier_v2: 0.8244763507490608\n",
      "ExtraTreesClassifier: 0.8257472968425423\n",
      "GradientBoostingClassifier: 0.8252509150745929\n",
      "LGBMClassifier: 0.8242357999459483\n",
      "XGBClassifier: 0.8230064578597459\n",
      "CatBoostClassifier: 0.8313917650438513\n"
     ]
    }
   ],
   "source": [
    "rf_v1_params = {\n",
    "    'n_jobs': -1,\n",
    "    'n_estimators': 50,\n",
    "     'warm_start': True, \n",
    "     #'max_features': 0.2,\n",
    "    'max_depth': 7,\n",
    "    'min_samples_leaf': 2,\n",
    "    'max_features' : 'sqrt',\n",
    "    'verbose': 0,\n",
    "    'random_state':2020,\n",
    "}\n",
    "rf_v2_params = {\n",
    "    'oob_score':True,\n",
    "    'random_state':2020,\n",
    "    'n_estimators': 70,\n",
    "    'max_depth':13,\n",
    "    'min_samples_split':5\n",
    "}\n",
    "# Extra Trees Parameters\n",
    "et_params = {\n",
    "    'n_jobs': -1,\n",
    "    'n_estimators':60,\n",
    "    'max_features': 0.5,\n",
    "    'max_depth': 20,\n",
    "    'min_samples_leaf': 1,\n",
    "    'verbose': 0,\n",
    "    'random_state':2020,\n",
    "}\n",
    "\n",
    "\n",
    "# Gradient Boosting parameters\n",
    "gb_params = {\n",
    "    'n_estimators': 60,\n",
    "    'max_features': 0.2,\n",
    "    'max_depth': 5,\n",
    "    'min_samples_leaf': 2,\n",
    "    'verbose': 0,\n",
    "    'random_state':2020,\n",
    "}\n",
    "#\n",
    "lgb_params = {'num_leaves':9\n",
    "                ,'max_depth':8\n",
    "                ,'learning_rate':0.05\n",
    "                ,'n_estimators':150\n",
    "                ,'n_jobs':8\n",
    "             }\n",
    "xgb_params ={'max_depth':7\n",
    "              ,'learning_rate':0.05\n",
    "              ,'n_estimators':55\n",
    "              ,'reg_alpha':0.005\n",
    "              ,'n_jobs':8\n",
    "              ,'importance_type':'total_cover'\n",
    "            }\n",
    "cab_params={'iterations':60\n",
    "          ,'learning_rate':0.05\n",
    "          ,'depth':10\n",
    "          ,'silent':True\n",
    "          ,'thread_count':8\n",
    "          ,'task_type':'CPU'\n",
    "          #,'cat_features':cat_features\n",
    "}\n",
    "#\n",
    "rf_v1 = RandomForestClassifier(**rf_v1_params)\n",
    "print(\"RandomForestClassifier_v1:\",k_fold_serachParmaters(rf_v1,train_data,kind))\n",
    "rf_v2 = RandomForestClassifier(**rf_v2_params)\n",
    "print(\"RandomForestClassifier_v2:\",k_fold_serachParmaters(rf_v2,train_data,kind))\n",
    "et = ExtraTreesClassifier(**et_params)\n",
    "print(\"ExtraTreesClassifier:\",k_fold_serachParmaters(et,train_data,kind))\n",
    "gb = GradientBoostingClassifier(**gb_params)\n",
    "print(\"GradientBoostingClassifier:\",k_fold_serachParmaters(gb,train_data,kind))\n",
    "lgb=lightgbm.LGBMClassifier(**lgb_params)\n",
    "print('LGBMClassifier:',k_fold_serachParmaters(lgb,train_data,kind))\n",
    "xgb=xgboost.XGBClassifier(**xgb_params)\n",
    "print('XGBClassifier:',k_fold_serachParmaters(xgb,train_data,kind))\n",
    "cab=catboost.CatBoostClassifier(**cab_params)\n",
    "print('CatBoostClassifier:',k_fold_serachParmaters(cab,train_data,kind))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "#\n",
    "def cnt_result(xx):\n",
    "    cnt_re={0:0,1:0}\n",
    "    for a in xx:\n",
    "        if a<=0.5:\n",
    "            cnt_re[0]+=1\n",
    "        else:\n",
    "            cnt_re[1]+=1\n",
    "    return cnt_re\n",
    "#"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 两层stacking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------->>>>>定义一个Sklearn classifier 的扩展<<<<<--------\n",
      "-------->>>>>定义oof:stacking的核心流程<<<<<--------\n",
      "-------->>>>>第一层的基模型<<<<<--------\n",
      "-------->>>>>训练、测试数据<<<<<--------\n",
      "y_train:(14865,);train_data:(14865, 72);test_data:(10000, 72)\n",
      "-------->>>>>第一阶段训练oof<<<<<--------\n",
      "Extra Trees.............\n",
      "Random Forest.............\n",
      "Random Forest_v2.............\n",
      "LGBClassifier.............\n",
      "Gradient.............\n",
      "XGBClassifier.............\n",
      "CatClassifier.............\n",
      "Training Stage_1 is complete\n"
     ]
    }
   ],
   "source": [
    "# \n",
    "use_selectFrt=False\n",
    "ntrain = train_df.shape[0]\n",
    "ntest = test_df.shape[0]\n",
    "SEED = 0 # for reproducibility\n",
    "NFOLDS = 5 # set folds for out-of-fold prediction\n",
    "#kf = KFold(n_splits= NFOLDS, random_state=SEED)\n",
    "kf=StratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=2020)\n",
    "print(\"-------->>>>>定义一个Sklearn classifier 的扩展<<<<<--------\")\n",
    "# Class to extend the Sklearn classifier\n",
    "class SklearnHelper(object):\n",
    "    def __init__(self, clf, seed=0, params=None):\n",
    "        params['random_state'] = seed\n",
    "        self.clf = clf(**params)\n",
    "\n",
    "    def train(self, x_train, y_train):\n",
    "        self.clf.fit(x_train, y_train)\n",
    "\n",
    "    def predict(self, x):\n",
    "        return self.clf.predict(x)\n",
    "    \n",
    "    def predict_proba(self, x):\n",
    "        return self.clf.predict_proba(x)[:,1]\n",
    "    \n",
    "    def fit(self,x,y):\n",
    "        return self.clf.fit(x,y)\n",
    "    \n",
    "    def feature_importances(self,x,y):\n",
    "        print(self.clf.fit(x,y).feature_importances_)\n",
    "#\"-----------------定义oof:stacking的核心流程-------------\"\n",
    "print(\"-------->>>>>定义oof:stacking的核心流程<<<<<--------\")\n",
    "'''\n",
    "'''\n",
    "def get_oof(clf, x_train, y_train, x_test):\n",
    "    oof_train = np.zeros((ntrain,))#(14865,)\n",
    "    oof_test = np.zeros((ntest,))#(10000,)\n",
    "    oof_test_skf = np.empty((NFOLDS, ntest))#(5, 10000)\n",
    "    #\n",
    "    fold=0\n",
    "    for train_index, test_index in kf.split(x_train, y_train):\n",
    "        x_tr = x_train[train_index]\n",
    "        y_tr = y_train[train_index]\n",
    "        x_te = x_train[test_index]#该折余下的验证集\n",
    "\n",
    "        clf.train(x_tr, y_tr)\n",
    "\n",
    "        oof_train[test_index] = clf.predict_proba(x_te)#对该折余下的验证集做预测，将结果填补在这些数据在原始数据中的位置\n",
    "        oof_test_skf[fold, :] = clf.predict_proba(x_test)#用此时的模型（第i折的模型）对测试做预测，放在第i折对应的位置\n",
    "        #oof_train[test_index] = clf.predict(x_te)\n",
    "        #oof_test_skf[fold, :] = clf.predict(x_test)\n",
    "        fold+=1\n",
    "\n",
    "    oof_test[:] = oof_test_skf.mean(axis=0)#将这N折模型对测试集的预测结果进行一个平均，作为改模型的预测结果\n",
    "    return oof_train.reshape(-1, 1), oof_test.reshape(-1, 1)\n",
    "#\n",
    "'''\n",
    "第一层的基模型\n",
    "'''\n",
    "# Put in our parameters for said classifiers\n",
    "# Random Forest parameters\n",
    "# \n",
    "print(\"-------->>>>>第一层的基模型<<<<<--------\")\n",
    "rf_v1 = SklearnHelper(clf=RandomForestClassifier, seed=SEED, params=rf_v1_params)\n",
    "rf_v2 = SklearnHelper(clf=RandomForestClassifier, seed=SEED, params=rf_v2_params)\n",
    "et = SklearnHelper(clf=ExtraTreesClassifier, seed=SEED, params=et_params)\n",
    "gb = SklearnHelper(clf=GradientBoostingClassifier, seed=SEED, params=gb_params)\n",
    "lgb= SklearnHelper(clf=lightgbm.LGBMClassifier, seed=SEED, params=lgb_params)\n",
    "xgb = SklearnHelper(clf=xgboost.XGBClassifier, seed=SEED, params=xgb_params)\n",
    "cab=SklearnHelper(clf=catboost.CatBoostClassifier, seed=SEED, params=cab_params)\n",
    "\n",
    "#\n",
    "# Create Numpy arrays of train, test and target ( Survived) dataframes to feed into our models\n",
    "#\n",
    "train_df=all_data.merge(entprise_info)\n",
    "test_df=all_data[all_data['id'].isin(entprise_evaluate['id'].unique().tolist())]\n",
    "test_df=test_df.reset_index(drop=True)\n",
    "y_train = train_df['label'].ravel()\n",
    "train_data = train_df.drop(['id','label'], axis=1)\n",
    "test_data=test_df.drop(['id'],axis=1)\n",
    "if use_selectFrt:\n",
    "    select_frt=important_frt[:30]\n",
    "    train_data=train_data[select_frt]\n",
    "    test_data=test_data[select_frt]\n",
    "x_train = train_data.values # Creates an array of the train data\n",
    "x_test = test_data.values # Creats an array of the test data\n",
    "#\n",
    "print(\"-------->>>>>训练、测试数据<<<<<--------\")\n",
    "print(\"y_train:{};train_data:{};test_data:{}\".format(y_train.shape,train_data.shape,test_data.shape))\n",
    "#\n",
    "# Create our OOF train and test predictions. These base results will be used as new features\n",
    "print(\"-------->>>>>第一阶段训练oof<<<<<--------\")\n",
    "et_oof_train, et_oof_test = get_oof(et, x_train, y_train, x_test) # Extra Trees\n",
    "print(\"Extra Trees.............\")\n",
    "rf_oof_train_v1, rf_oof_test_v1 = get_oof(rf_v1,x_train, y_train, x_test) # Random Forest\n",
    "print(\"Random Forest_v1.............\")\n",
    "rf_oof_train_v2, rf_oof_test_v2 = get_oof(rf_v2,x_train, y_train, x_test) # Random Forest\n",
    "print(\"Random Forest_v2.............\")\n",
    "lgb_oof_train, lgb_oof_test = get_oof(lgb,x_train, y_train, x_test) # Gradient Boost\n",
    "print(\"LGBClassifier.............\")\n",
    "gb_oof_train, gb_oof_test = get_oof(gb,x_train, y_train, x_test) # Gradient Boost\n",
    "print(\"Gradient.............\")\n",
    "xgb_oof_train, xgb_oof_test = get_oof(xgb,x_train, y_train, x_test) # XGBClassifier\n",
    "print(\"XGBClassifier.............\")\n",
    "cab_oof_train, cab_oof_test = get_oof(cab,x_train, y_train, x_test)# CatClassifier\n",
    "print(\"CatClassifier.............\")\n",
    "\n",
    "print(\"Training Stage_1 is complete\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# #\n",
    "x_train = np.concatenate(( et_oof_train, rf_oof_train_v1, rf_oof_train_v2, gb_oof_train, lgb_oof_train, xgb_oof_train,cab_oof_train), axis=1)\n",
    "x_test = np.concatenate(( et_oof_test, rf_oof_test_v1,rf_oof_test_v2, gb_oof_test,lgb_oof_test, xgb_oof_test, cab_oof_test), axis=1)\n",
    "#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n_estimators: 80\n",
      "[80, 0.04, 4, 4] 0.8516515249707891\n",
      "[80, 0.04, 4, 5] 0.8518063396655999\n",
      "[80, 0.04, 4, 6] 0.854340990298924\n",
      "[80, 0.04, 4, 8] 0.8555600899834916\n",
      "[80, 0.04, 5, 8] 0.8561099582451429\n"
     ]
    }
   ],
   "source": [
    "def search_param(n_estimators,lr,max_depth,num_leaves):\n",
    "    params={'num_leaves':num_leaves\n",
    "                   ,'max_depth':max_depth\n",
    "                   ,'learning_rate':lr\n",
    "                   ,'n_estimators':n_estimators\n",
    "                   ,'n_jobs':8\n",
    "               }\n",
    "    mean_f1=k_fold_serachParmaters(lightgbm.LGBMClassifier(**params),pd.DataFrame(x_train),pd.DataFrame(y_train))\n",
    "    return mean_f1\n",
    "\n",
    "#搜索最佳参数\n",
    "param=[]\n",
    "best=0\n",
    "for n_estimators in [80]:\n",
    "    print('n_estimators:',n_estimators)\n",
    "    for lr in [0.04,0.05,0.06]:\n",
    "        for max_depth in [4,5,6,7,8]:\n",
    "            for num_leaves in [4,5,6,7,8]:\n",
    "                mean_f1=search_param(n_estimators,lr,max_depth,num_leaves)\n",
    "                if mean_f1>best:\n",
    "                    param=[n_estimators,lr,max_depth,num_leaves]\n",
    "                    best=mean_f1\n",
    "                    print(param,best)#[80, 0.05, 4, 4] 0.8419565786981478"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n_estimators: 50\n",
      "[50, 0.03, 3, 5] 0.8521864513401066\n",
      "[50, 0.03, 3, 11] 0.8523042631007035\n",
      "[50, 0.03, 3, 13] 0.8537931384951206\n",
      "[50, 0.03, 6, 13] 0.8558096475945506\n"
     ]
    }
   ],
   "source": [
    "def search_param(n_estimators,lr,max_depth,min_child_weight):\n",
    "    params={'max_depth':max_depth\n",
    "              ,'learning_rate':lr\n",
    "              ,'n_estimators':n_estimators\n",
    "              ,'min_child_weight':min_child_weight\n",
    "              ,'n_jobs':8\n",
    "              ,'importance_type':'total_cover'\n",
    "           }\n",
    "    \n",
    "    mean_f1=k_fold_serachParmaters(xgboost.XGBClassifier(**params),pd.DataFrame(x_train),pd.DataFrame(y_train))\n",
    "    return mean_f1\n",
    "\n",
    "#搜索最佳参数\n",
    "param=[]\n",
    "best=0\n",
    "for n_estimators in [50]:\n",
    "    print('n_estimators:',n_estimators)\n",
    "    for lr in [0.03,0.04]:\n",
    "        for max_depth in [3,4,5,6]:\n",
    "            for min_child_weight in [5,7,9,11,13,15]:\n",
    "                mean_f1=search_param(n_estimators,lr,max_depth,min_child_weight)\n",
    "                if mean_f1>best:\n",
    "                    param=[n_estimators,lr,max_depth,min_child_weight]\n",
    "                    best=mean_f1\n",
    "                    print(param,best)#[50, 0.04, 4, 13] 0.8449774193716448"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "iter_cnt: 100\n",
      "[100, 0.03, 7] 0.8480191309321614\n",
      "[100, 0.03, 8] 0.8481074193605483\n",
      "[100, 0.03, 6] 0.8492033735787374\n",
      "[100, 0.04, 7] 0.8506759802228252\n",
      "[100, 0.04, 8] 0.8509362237864766\n",
      "[100, 0.05, 7] 0.8543198637162136\n",
      "[100, 0.05, 6] 0.8546795944699905\n",
      "[100, 0.05, 6] 0.8546795944699905\n"
     ]
    }
   ],
   "source": [
    "def search_param(iter_cnt,lr,max_depth):\n",
    "    \n",
    "    params={'iterations':iter_cnt\n",
    "              ,'learning_rate':lr\n",
    "              ,'depth':max_depth\n",
    "              ,'silent':True\n",
    "              ,'thread_count':8\n",
    "              ,'task_type':'CPU'\n",
    "        }\n",
    "    mean_f1=k_fold_serachParmaters(catboost.CatBoostClassifier(**params),pd.DataFrame(x_train),pd.DataFrame(y_train))\n",
    "    return mean_f1\n",
    "\n",
    "#搜索最佳参数[52, 0.073, 7] 0.8440488273317126\n",
    "param=[]\n",
    "best=0\n",
    "for iter_cnt in [100]:\n",
    "    print('iter_cnt:',iter_cnt)\n",
    "    for lr in [0.03,0.04,0.05]:\n",
    "        for max_depth in [7,8,6]:\n",
    "            mean_f1=search_param(iter_cnt,lr,max_depth)\n",
    "            if mean_f1>best:\n",
    "                param=[iter_cnt,lr,max_depth]#[55, 0.04, 5] 0.8475966668074264\n",
    "                best=mean_f1\n",
    "                print(param,best)\n",
    "print(param,best)#[55, 0.04, 5] 0.8475966668074264\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n_estimators: 40\n",
      "[40, 4, 4] 0.8500048665554116\n",
      "[40, 7, 4] 0.8514957600585706\n",
      "[40, 9, 4] 0.8530427606351101\n",
      "n_estimators: 50\n",
      "[50, 9, 7] 0.8541367972490226\n"
     ]
    }
   ],
   "source": [
    "# #\n",
    "def search_param(n_estimators,max_depth,min_samples_split):\n",
    "    params={'oob_score':True,\n",
    "            'random_state':2020,\n",
    "            'n_estimators': n_estimators,\n",
    "            'max_depth':max_depth,\n",
    "            'min_samples_split':min_samples_split\n",
    "            }\n",
    "    mean_f1=k_fold_serachParmaters(RandomForestClassifier(**params),pd.DataFrame(x_train),pd.DataFrame(y_train))\n",
    "    return mean_f1\n",
    "\n",
    "#搜索最佳参数\n",
    "param=[]\n",
    "best=0\n",
    "for n_estimators in [40,50]:\n",
    "    print('n_estimators:',n_estimators)\n",
    "    for min_samples_split in [4,5,7,8]:\n",
    "        for max_depth in [4,5,7,9,]:\n",
    "            mean_f1=search_param(n_estimators,max_depth,min_samples_split)\n",
    "            if mean_f1>best:\n",
    "                param=[n_estimators,max_depth,min_samples_split]\n",
    "                best=mean_f1\n",
    "                print(param,best)#[50, 4, 8] 0.8442795786278773\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# stage-2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LGBMClassifier: 0.8561099582451429\n",
      "XGBClassifier: 0.8558096475945506\n",
      "CatBoostClassifier: 0.8546795944699905\n",
      "RandomForestClassifier_v1: 0.8541367972490226\n"
     ]
    }
   ],
   "source": [
    "#stage_2的模型调参\n",
    "\n",
    "lgb_params_stage2 = {'num_leaves':8\n",
    "                ,'max_depth':5\n",
    "                ,'learning_rate':0.04\n",
    "                ,'n_estimators':80\n",
    "                ,'n_jobs':8\n",
    "             }\n",
    "xgb_params_stage2 ={'max_depth':6\n",
    "              ,'learning_rate':0.03\n",
    "              ,'n_estimators':50\n",
    "              ,'min_child_weight':13\n",
    "              ,'n_jobs':8\n",
    "              ,'importance_type':'total_cover'\n",
    "            }\n",
    "cab_params_stage2={'iterations':100\n",
    "          ,'learning_rate':0.05\n",
    "          ,'depth':6\n",
    "          ,'silent':True\n",
    "          ,'thread_count':8\n",
    "          ,'task_type':'CPU'\n",
    "          #,'cat_features':cat_features\n",
    "}\n",
    "rf_params_stage2={'oob_score':True,\n",
    "            'random_state':2020,\n",
    "            'n_estimators': 50,\n",
    "            'max_depth':9,\n",
    "            'min_samples_split':7\n",
    "            }\n",
    "# #\n",
    "x_train = np.concatenate(( et_oof_train, rf_oof_train_v1, rf_oof_train_v2, gb_oof_train, lgb_oof_train, xgb_oof_train,cab_oof_train), axis=1)\n",
    "x_test = np.concatenate(( et_oof_test, rf_oof_test_v1,rf_oof_test_v2, gb_oof_test,lgb_oof_test, xgb_oof_test, cab_oof_test), axis=1)\n",
    "#\n",
    "lgb_stage2=lightgbm.LGBMClassifier(**lgb_params_stage2)\n",
    "print('LGBMClassifier:',k_fold_serachParmaters(lgb_stage2,pd.DataFrame(x_train),pd.DataFrame(y_train)))\n",
    "xgb_stage2=xgboost.XGBClassifier(**xgb_params_stage2)\n",
    "print('XGBClassifier:',k_fold_serachParmaters(xgb_stage2,pd.DataFrame(x_train),pd.DataFrame(y_train)))\n",
    "cab_stage2=catboost.CatBoostClassifier(**cab_params_stage2)\n",
    "print('CatBoostClassifier:',k_fold_serachParmaters(cab_stage2,pd.DataFrame(x_train),pd.DataFrame(y_train)))\n",
    "rf_stage2 = RandomForestClassifier(**rf_params_stage2)\n",
    "print(\"RandomForestClassifier_v1:\",k_fold_serachParmaters(rf_stage2,pd.DataFrame(x_train),pd.DataFrame(y_train)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 训练stage2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training is complete\n",
      "------第二阶段训练开始--------\n",
      "预测结果中合法的数量9109;违法的数量891,合法/违法10.223345\n",
      "预测结果中合法的数量9104;违法的数量896,合法/违法10.160714\n",
      "预测结果中合法的数量9094;违法的数量906,合法/违法10.037528\n",
      "预测结果中合法的数量9093;违法的数量907,合法/违法10.025358\n",
      "最终预测结果中合法的数量9104;违法的数量896,合法/违法10.160714\n"
     ]
    }
   ],
   "source": [
    "print(\"Training is complete\")\n",
    "print(\"------第二阶段训练开始--------\")\n",
    "x_train = np.concatenate(( et_oof_train, rf_oof_train, ada_oof_train, gb_oof_train, xgb_oof_train,cab_oof_train), axis=1)\n",
    "x_test = np.concatenate(( et_oof_test, rf_oof_test, ada_oof_test,gb_oof_test, xgb_oof_test, cab_oof_test), axis=1)\n",
    "\n",
    "#\n",
    "predictions_stage2=[]\n",
    "for model_two_stage in [lgb_stage2,xgb_stage2,cab_stage2,rf_stage2]:\n",
    "    #\n",
    "    model_two_stage.fit(x_train, y_train)\n",
    "    predictions = model_two_stage.predict_proba(x_test)[:,1]\n",
    "    predictions_stage2.append(predictions)\n",
    "    cnt_re=cnt_result(predictions)\n",
    "    print(\"预测结果中合法的数量%d;违法的数量%d,合法/违法%f\"%(cnt_re[0],cnt_re[1],cnt_re[0]/cnt_re[1]))\n",
    "#\n",
    "predictions=sum(predictions_stage2)/len(predictions_stage2)#\n",
    "#predictions=np.sqrt(sum(np.array(np.array(predictions_stage2))**2)/len(predictions_stage2))#平方平均\n",
    "#predictions=pow(np.prod(np.array(predictions_stage2), axis=0),1/len(predictions_stage2))#几何平均\n",
    "cnt_re=cnt_result(predictions)\n",
    "print(\"最终预测结果中合法的数量%d;违法的数量%d,合法/违法%f\"%(cnt_re[0],cnt_re[1],cnt_re[0]/cnt_re[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "投票结果中合法的数量9097;违法的数量903,合法/违法10.074197\n"
     ]
    }
   ],
   "source": [
    "votes=[(pre>0.5).astype(int) for pre in predictions_stage2]\n",
    "vote_most=[]\n",
    "for i in range(len(predictions_stage2[0])):\n",
    "    vote_list=np.array(votes)[:,i]\n",
    "    tmp = {0: 0, 1: 0}\n",
    "    for k in vote_list:\n",
    "        tmp[k] += 1\n",
    "    #\n",
    "    most = sorted(tmp.items(), key=lambda item: item[1])[-1][0]\n",
    "    vote_most.append(most)\n",
    "cnt_re=cnt_result(vote_most)\n",
    "print(\"投票结果中合法的数量%d;违法的数量%d,合法/违法%f\"%(cnt_re[0],cnt_re[1],cnt_re[0]/cnt_re[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9989"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score((predictions>0.5).astype(int),vote_most)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'../submission/submit_stack_1120_903.csv'"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "test_df['score']=vote_most#predictions#\n",
    "submit_csv=test_df[['id','score']]\n",
    "save_path='../submission/submit_stack_1120_'+str(cnt_re[1])+'.csv'\n",
    "submit_csv.to_csv(save_path,index=False)\n",
    "save_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9c7fa510616a683058ce97d0bc768a621cd85ab1e87da2a3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>da8691b210adb3f67820f5e0c87b337d63112cee52211888</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9c7fa510616a68309e4badf2a7a3123c0462fb85bf28ef17</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>f000950527a6feb6ed308bc4c7ae11276eab86480f8e03db</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>f000950527a6feb617e8d6ca7025dcf9d765429969122069</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9995</th>\n",
       "      <td>f1c1045b13d18329a2bd99d2a7e2227688c0d69bf1d1e325</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9996</th>\n",
       "      <td>f000950527a6feb6bde38216d7cbbf32e66d3a3a96d4dbda</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9997</th>\n",
       "      <td>da8691b210adb3f65b43370d3a362f4aa1d3b16b5ba0c9d7</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9998</th>\n",
       "      <td>516ab81418ed215dcbbf0614a7b929e691f8eed153d7bb31</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9999</th>\n",
       "      <td>9c7fa510616a68303d3427d4bfd4b0cf3e4843f2bf3f637a</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10000 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    id  score\n",
       "0     9c7fa510616a683058ce97d0bc768a621cd85ab1e87da2a3      0\n",
       "1     da8691b210adb3f67820f5e0c87b337d63112cee52211888      0\n",
       "2     9c7fa510616a68309e4badf2a7a3123c0462fb85bf28ef17      0\n",
       "3     f000950527a6feb6ed308bc4c7ae11276eab86480f8e03db      0\n",
       "4     f000950527a6feb617e8d6ca7025dcf9d765429969122069      0\n",
       "...                                                ...    ...\n",
       "9995  f1c1045b13d18329a2bd99d2a7e2227688c0d69bf1d1e325      0\n",
       "9996  f000950527a6feb6bde38216d7cbbf32e66d3a3a96d4dbda      1\n",
       "9997  da8691b210adb3f65b43370d3a362f4aa1d3b16b5ba0c9d7      0\n",
       "9998  516ab81418ed215dcbbf0614a7b929e691f8eed153d7bb31      0\n",
       "9999  9c7fa510616a68303d3427d4bfd4b0cf3e4843f2bf3f637a      0\n",
       "\n",
       "[10000 rows x 2 columns]"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submit_csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "huawei_copy",
   "language": "python",
   "name": "huawei_copy"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
